#ifndef __FAST_DEDUP_DOT_HPP__
#define __FAST_DEDUP_DOT_HPP__
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
#include <zest.hpp>
#include <memline.hpp>
#include <IndexedSet.hpp>
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
#include <map>
#include <string>
#include <vector>
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
#include <Timer.hpp>
#include <BaseDedup.hpp>
#include <HashFunctions.hpp>
#include <DRAMImageStream.hpp>
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
#include <memline.hpp>
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
class FastDedupMaster;
class FastDedupThread : public BaseDedupThread {
    
protected:
    FastDedupThread * sibling;
    FastDedupMaster * parent;
    
public:
    FastDedupThread( FastDedupMaster * _parent, const uint32_t _tidx, HashWrapper * _hf, pthread_mutex_t * _disklock, const bool _sync ) :
        BaseDedupThread( ((BaseDedupMaster*)(_parent)), _tidx, _hf, _disklock, _sync ),
        parent    ( _parent )
    {}
    
    void LineDedup( const uint64_t currPassIdx );
    void PageDedup( const uint64_t currPassIdx );
    void HugeDedup( const uint64_t currPassIdx );
    void SetFICs( DleftIndexedCounters * _fg, DleftIndexedCounters * _pg, DleftIndexedCounters * _hg ) {
        fg = _fg;
        pg = _pg;
        hg = _hg;
    }
};
class FastDedupMaster : public BaseDedupMaster {

    JenkinsHash * hf0;
    JenkinsHash * hf1;

    static const uint64_t numPasses = 1;

    std::vector< FastDedupThread * > fddthreads;
    
    void FastDedup( void );
    void DeallocKeysAndSigs( void );
public:
    FastDedupMaster( const uint32_t _numChunks, const uint32_t _numThreads, const char * mipath, const char * rcpath, const bool _impl ) : BaseDedupMaster( numPasses, _numChunks, _numThreads, mipath, rcpath, _impl ) {
        
        const uint64_t bytes = SumMemImageChunkSizes();
        const uint64_t numlines   = bytes / 64ULL;
        const uint64_t numpages   = bytes / 4096ULL;
        const uint64_t numhuges   = bytes / uint64_t( 2 * 1024 * 1024 );
        const uint64_t fg_buckets = numlines  >> 6;
        const uint64_t pg_buckets = numpages  >> 6;
        const uint64_t hg_buckets = numhuges  >> 6;
        const uint32_t fgIdxBits  = clog2( fg_buckets ) > 4 ? clog2( fg_buckets ) : 4;
        const uint32_t pgIdxBits  = clog2( pg_buckets ) > 4 ? clog2( pg_buckets ) : 4;
        const uint32_t hgIdxBits  = clog2( hg_buckets ) > 4 ? clog2( hg_buckets ) : 4;
        
        // cout << "gigabytes = " << double( bytes ) / 1024.0 / 1024.0 / 1024.0 << endl;
        // cout << "numlines  = " << numlines << endl;
        // cout << "numpages  = " << numpages << endl;
        // cout << "numhuges  = " << numhuges << endl;
        // cout << "fgIdxBits = " << fgIdxBits << endl;
        // cout << "pgIdxBits = " << pgIdxBits << endl;
        // cout << "hgIdxBits = " << hgIdxBits << endl;
        // 
        
        const uint32_t wordsPerKey = 1;
        fg = new DleftIndexedCounters( fgIdxBits, wordsPerKey );
        pg = new DleftIndexedCounters( pgIdxBits, wordsPerKey );
        hg = new DleftIndexedCounters( hgIdxBits, wordsPerKey );
        
        fg->SetName( "fg" );
        pg->SetName( "pg" );
        hg->SetName( "hg" );
        
        hf0 = new JenkinsHash( 0xb492b66fbe98f273ULL );
        hf1 = new JenkinsHash( 0xc3a5c85c97cb3127ULL );
        
        fg->SetHashes( hf0, hf1 );
        pg->SetHashes( hf0, hf1 );
        hg->SetHashes( hf0, hf1 );

        // use this to test hash table overflows:
        // ... each bucket has a capacity of 64
        // ... PreLoad( n ) inserts n elements per bucket
        // fg->PreLoad( 45 );
        // pg->PreLoad( 45 );
        // hg->PreLoad( 45 );

        // disk i/o optimization options; defaults:
        // ... do not lock call to read(); alternate: plock = &disklock
        // ... use asynch read();          alternate: sync = true calls open() with O_SYNC
        pthread_mutex_t * plock = NULL;
        const bool sync = false;
        
        for( uint32_t tidx = 0; tidx < numThreads; tidx++ ) {

            FastDedupThread * dd = new FastDedupThread( this, tidx, &jhf, plock, sync );
            dd->SetFICs( fg, pg, hg );
            
            fddthreads.push_back( dd );
            bddthreads.push_back( dd );
        }
    }
    ~FastDedupMaster( void ) {
        for( uint32_t tidx = 0; tidx < numThreads; tidx++ ) {
            delete fddthreads[ tidx ];
        }
        delete fg;
        delete pg;
        delete hg;
        delete hf0;
        delete hf1;
    }
    void RunDedup( void );
};
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
#endif

